package ruleGeneration;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.text.DecimalFormat;
import java.text.DecimalFormatSymbols;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.Map.Entry;

import ruleGeneration.GenerateRules.Rule;

import mining.CsvReader;
import fpTree.FListComparator;
import fpTree.Node;
import fpTree.TreeMethods;

public class GenerateRulesRareConsequent
{
	public static void main(String[] args)
	{
//		GenerateRulesRareConsequent r = new GenerateRulesRareConsequent("./datasets/connect4/connect-4.data.modified-5-10134-RpTreeInfoGain", 0.9, 1.0, false);
//		r.run();
		
		runLots();
	}

	private static void runLots()
	{
		String[] files = new String[]{
				"./datasets/connect4/connect-4.data.modified-0.1-5-10134-RpTreeInfoGain",
				"./datasets/voting_records/house-votes-84.data.modified-0.1-5-66-RpTreeInfoGain",
				"./datasets/spect/SPECT.all.modified-0.1-5-41-RpTreeInfoGain",
				"./datasets/primary_tumor/primary-tumor.data.modified-0.1-5-51-RpTreeInfoGain",
				"./datasets/zoo/zoo.data.modified-0.1-5-16-RpTreeInfoGain",
				"./datasets/tae/tae.data.modified-0.1-5-10-RpTreeInfoGain",
				"./datasets/flags/flags.data.modified-0.1-5-30-RpTreeInfoGain",
				"./datasets/adult/adult.all.modified-0.1-5-7327-RpTreeInfoGain",
				"./datasets/dermatology/dermatology.data.modified-0.1-20-55-RpTreeInfoGain",
				"./datasets/soybean_large/soybean-large.data.modified-0.1-40-47-RpTreeInfoGain",
				"./datasets/mushroom/agaricus-lepiota.data.modified-0.1-40-1219-RpTreeInfoGain", 
		};
		
		GenerateRulesRareConsequent r;
		double minConf = 0.9;
		double minLift = 1.0;
		for (String file : files)
		{
			r = new GenerateRulesRareConsequent(file, minConf, minLift, false);
			r.run();
		}
	}

	private BufferedWriter ruleWriter;
	private BufferedWriter record;
	
	long time = 0;
	long writeTime = 0;
	
	String itemsetFile;
	String transactionFile;
	String ruleOutputFile;
	
	private DecimalFormat decimalFormat;
	private double minConf = 0.9;
	private double minLift = 1.0;
	private boolean rareConsequentOnly;
	public GenerateRulesRareConsequent(String itemsetLocation, double minConf, double minLift, boolean rareConsequentOnly) {
		this.itemsetFile = itemsetLocation;
		this.transactionFile = itemsetLocation.substring(0, itemsetLocation.indexOf(".modified") + 9);
		this.ruleOutputFile = itemsetLocation + "." + Double.toString(minConf) + "-" + Double.toString(minLift) + "-" + this.rareConsequentOnly + ".rule";
		System.out.println(this.itemsetFile);
		System.out.println(this.transactionFile);
		System.out.println(this.ruleOutputFile);

		this.minConf = minConf;
		this.minLift = minLift;
		this.rareConsequentOnly = rareConsequentOnly;
		try
		{
			this.record = new BufferedWriter(new FileWriter("./RuleGenerationRecord", true));
		} catch (IOException e)
		{
			e.printStackTrace();
		}
		
		DecimalFormatSymbols dfs = new DecimalFormatSymbols();
		dfs.setInfinity("inf");
		this.decimalFormat = new DecimalFormat("#0.000", dfs);
	}

	public void run()
	{

		TreeObj treeObj = buildTree(this.transactionFile);
//		System.out.println(treeObj.root.printTree());
		System.out.println(treeObj.itemFreqs);
		System.out.println(treeObj.headerTable);
		
//		readFileItemsets(new File(this.itemsetFile), treeObj);
		readAllItemsets(this.itemsetFile, treeObj);

		
//		int sup = countSupport3(new ArrayList<Integer>(Arrays.asList(5,6)), treeObj.headerTable);
//		System.out.println("support: " + sup);
		
	}
	
	public void readAllItemsets(String itemsetLocation, TreeObj treeObj) {
		long t1 = System.nanoTime();
		
		File file = new File(itemsetLocation);
		
		try
		{
			FileWriter fw = new FileWriter(this.ruleOutputFile);
			this.ruleWriter = new BufferedWriter(fw);
		} catch (IOException e1)
		{
			e1.printStackTrace();
		}
		
		readFileItemsets(file, treeObj);
		
		for (int i = 1;; i++)
		{
			System.out.println("processing file: " + itemsetLocation + "." + i);
			File nextFile = new File(itemsetLocation + "." + i);
			if (nextFile.exists())
			{
				readFileItemsets(nextFile, treeObj);
			} else {
				System.out.println("does not exist. terminating.");
				break;
			}
		}

		long t2 = System.nanoTime();
		
		this.time += (t2 - t1 - this.writeTime) / 1000000;

		try
		{
			this.record.write(new SimpleDateFormat("dd/MM-HH:mm:ss").format(new Date()));
			this.record.write(",");
			this.record.write(new StringBuilder().append("\"").append(this.itemsetFile).append("\"").append(",").append(this.minConf).append(",")
					.append(this.rulesGenerated).append(",").append(this.rulesWritten).append(",").append(this.time)
					.append(",").toString());
			StringBuilder totals = new StringBuilder();
			this.getTotals(totals, this.rulesWritten);
			this.ruleWriter.append(totals.toString());
			this.record.append(totals.toString());
			
			this.ruleWriter.close();
			this.record.newLine();
			this.record.close();
		} catch (IOException e)
		{
			e.printStackTrace();
		}
	}
	
	List<BitSet> itemsets = new ArrayList<BitSet>(); 
	public void readFileItemsets(File itemsetLocation, TreeObj treeObj) {
		try
		{
			StringBuilder heading = new StringBuilder();
			writeHeading(heading, treeObj.transactionNum);
			this.ruleWriter.append(heading.toString());

//			int i = 0;
			BufferedReader rareBr = new BufferedReader(new FileReader(itemsetLocation));
			while (rareBr.ready())
			{
				String line = rareBr.readLine().replaceAll(" |(\\[)|(\\])", "");
				if (line.isEmpty())
					continue;
//				System.out.println(line);
				List<Rule> rules = generateRules(makeBitSet(line.substring(0, line.lastIndexOf(","))), treeObj.rares);
				int supAB = Integer.parseInt(line.substring(line.lastIndexOf(",") + 1, line.length()));

//				i++;
//				if (i % 1000 == 0)
//				{
//					System.out.println(i);
////					if (i == 3000)
////					{
////						System.exit(-1);
////					}
//				}
////				System.out.println("supAB: " + supAB);

				Comparator<Integer> c = new FListComparator(treeObj.itemFreqs);
				for (Rule rule : rules)
				{
					if (rule.a.isEmpty())
						continue;
					int supA = countSupport3(convertAndSort(rule.a, c), treeObj.headerTable);
					int supB = treeObj.itemFreqs.get(rule.b);
					
					writeAll(rule, supA, supB, supAB, treeObj.transactionNum);
					
//					System.out.println(rule + " | " + supA + ", " + supB + ", " + supAB);
				}
			}
			
		} catch (FileNotFoundException e)
		{
			e.printStackTrace();
		} catch (IOException e)
		{
			e.printStackTrace();
		}
	}
	
	private List<Integer> convertAndSort(BitSet a, Comparator<Integer> c)
	{
		List<Integer> converted = new ArrayList<Integer>();
		for(int i = a.nextSetBit(0); i >= 0; i = a.nextSetBit(i + 1))
		{
			converted.add(i);
		}
		Collections.sort(converted, c);
		return converted;
	}
	
	/**
	 * 
	 * @param a must be sorted in decreasing freq order
	 */
	private int countSupport3(List<Integer> a, Map<Integer, Node> headerTable) {
		int lastItem = a.remove(a.size() - 1);
		
		int support = 0;
		for (Node node = headerTable.get(lastItem); node != null; node = node.getNodeLink())
		{
				if (TreeMethods.existsInAncestors(node, a));
				{
					support += node.getCount();
				}
		}
		
		return support;
	}

	int numItems;
	private BitSet makeBitSet(String line)
	{
		Scanner scanner = new Scanner(line);
		scanner.useDelimiter(",");
		
		BitSet bitSet = new BitSet();

		while (scanner.hasNextInt())
		{
			int item = scanner.nextInt();
			this.numItems = Math.max(this.numItems, item + 1); 
			bitSet.set(item);
		}
		return bitSet;
	}
	
	class Rule {
		Rule(BitSet a, int b) {
			this.a = a;
			this.b = b;
		}
		BitSet a;
		int b;
		@Override
		public String toString()
		{
			return this.a + " -> " + this.b;
		}
	}
	
	private List<Rule> generateRules(BitSet itemset, BitSet rares) {
		List<Rule> rules = new ArrayList<Rule>();
		
		for(int i = itemset.nextSetBit(0); i >= 0; i = itemset.nextSetBit(i + 1))
		{
			if (!this.rareConsequentOnly || rares.get(i)) // consequent is a rare item
			{
				BitSet a = (BitSet) itemset.clone();
				a.set(i, false);
				rules.add(new Rule(a, i));
			}
		}
		
		return rules;
	}
	
	private void writeHeading(StringBuilder sb, int tNum) throws IOException {
		sb.append("a, b, supA, supB, supAB, conf, chisq, lift, allConf, coherence, cosine, kulc, allConf, transactions: ");
		sb.append(Integer.toString(tNum));
		sb.append("\n");
	}
	
	int rulesGenerated;
	int rulesWritten;
	int ruleWriterNum;
	private void writeAll(Rule rule, int supA, int supB, int supAB, int transactions) throws IOException
	{
		this.rulesGenerated++;
		double conf = conf(supA, supAB); 
		double lift = lift(supA, supB, supAB, transactions);
		if (conf < this.minConf || lift < this.minLift)
		{
//			System.out.println("conf too low: " + Double.toString(conf));
			return;
		}

		StringBuilder sb = new StringBuilder();
		writeRule(sb, rule, supA, supB, supAB);
		calcAndWriteMeasures(sb, supA, supB, supAB, transactions);
		
		// record time for writing to file...
		long t1 = System.nanoTime();

		this.rulesWritten++;
		if (this.rulesWritten % (1 << 22) == 0)
		{
			this.ruleWriterNum++;
			this.ruleWriter.close();
			this.ruleWriter = new BufferedWriter(new FileWriter(this.ruleOutputFile + "." + this.ruleWriterNum));
		}
		
		this.ruleWriter.append(sb.toString());
		this.writeTime += System.nanoTime() - t1;
	}
	
	// TODO: switch rule writer if too many rules...
	private void writeRule(StringBuilder sb, Rule rule, int supA, int supB, int supAB)
	{
		sb.append(rule.a.toString());
		sb.append(",");
		sb.append(Integer.toString(rule.b));
		sb.append(",");
		sb.append(Integer.toString(supA));
		sb.append(",");
		sb.append(Integer.toString(supB));
		sb.append(",");
		sb.append(Integer.toString(supAB));
		sb.append(",");
	}
	
	private double supT, confT, chisqT, 
		liftT, allConfT, 
		coherenceT, cosineT, 
		kulcT, maxConfT;
	private double supM = Double.MIN_VALUE, confM = Double.MIN_VALUE, chisqM = Double.MIN_VALUE, 
		liftM = Double.MIN_VALUE, allConfM = Double.MIN_VALUE, 
		coherenceM = Double.MIN_VALUE, cosineM = Double.MIN_VALUE, 
		kulcM, maxConfM = Double.MIN_VALUE;
	private double supm = Double.MAX_VALUE, confm = Double.MAX_VALUE, chisqm = Double.MAX_VALUE, 
		liftm = Double.MAX_VALUE, allConfm = Double.MAX_VALUE, 
		coherencem = Double.MAX_VALUE, cosinem = Double.MAX_VALUE, 
		kulcm = Double.MAX_VALUE, maxConfm = Double.MAX_VALUE;
	private void calcAndWriteMeasures(StringBuilder sb, int a, int b, int ab, int t) {
		this.supT += ab;
		this.supM = Math.max(this.supM, ab);
		this.supm = Math.min(this.supm, ab);
		
		double conf = conf(a, ab);
		this.confT += conf;
		this.confM = Math.max(this.confM, conf);
		this.confm = Math.min(this.confm, conf);
		writeDoubles(sb, conf);
		sb.append(",");
		
		double chisq = chisq(a, b, ab, t);
		if (chisq != Double.POSITIVE_INFINITY)
			this.chisqT += chisq;
		this.chisqM = Math.max(this.chisqM, chisq);
		this.chisqm = Math.min(this.chisqm, chisq);
		writeDoubles(sb, chisq);
		sb.append(",");
		
		double lift = lift(a, b, ab, t);
		if (lift != Double.POSITIVE_INFINITY)
			this.liftT += lift;
		this.liftM = Math.max(this.liftM, lift);
		this.liftm = Math.min(this.liftm, lift);
		writeDoubles(sb, lift);
		sb.append(",");
		
		double allConf = allConf(a, b, ab);
		this.allConfT += allConf;
		this.allConfM = Math.max(this.allConfM, allConf);
		this.allConfm = Math.min(this.allConfm, allConf);
		writeDoubles(sb, allConf);
		sb.append(",");
		
		double coherence = coherence(a, b, ab);
		this.coherenceT += coherence;
		this.coherenceM = Math.max(this.coherenceM, coherence);
		this.coherencem = Math.min(this.coherencem, coherence);
		writeDoubles(sb, coherence);
		sb.append(",");
		
		double cosine = cosine(a, b, ab);
		this.cosineT += cosine;
		this.cosineM = Math.max(this.cosineM, cosine);
		this.cosinem = Math.min(this.cosinem, cosine);
		writeDoubles(sb, cosine);
		sb.append(",");
		
		double kulc = kulc(a, b, ab);
		this.kulcT += kulc;
		this.kulcM = Math.max(this.kulcM, kulc);
		this.kulcm = Math.min(this.kulcm, kulc);
		writeDoubles(sb, kulc);
		sb.append(",");
		
		double maxConf = maxConf(a, b, ab);
		this.maxConfT += maxConf;
		this.maxConfM = Math.max(this.maxConfM, maxConf);
		this.maxConfm = Math.min(this.maxConfm, maxConf);
		writeDoubles(sb, maxConf);
		sb.append("\n");
	}

	private void getTotals(StringBuilder sb, int r) {
		writeDoubles(sb, this.supT/r, this.confT/r, this.chisqT/r, this.liftT/r, this.allConfT/r, this.coherenceT/r, this.cosineT/r, this.kulcT/r, this.maxConfT/r);
		sb.append(",");
		writeDoubles(sb, this.supM, this.confM, this.chisqM, this.liftM, this.allConfM, this.coherenceM, this.cosineM, this.kulcM, this.maxConfM);
		sb.append(",");
		writeDoubles(sb, this.supm, this.confm, this.chisqm, this.liftm, this.allConfm, this.coherencem, this.cosinem, this.kulcm, this.maxConfm);
	}

	private void writeDoubles(StringBuilder sb, double... ds) {
//		if (ds.length > 0)
//		{
		sb.append(this.decimalFormat.format(ds[0]));
//		}
		for(int i = 1; i < ds.length; i++)
		{
			sb.append(",");
			sb.append(this.decimalFormat.format(ds[i]));
		}
	}
	
	public TreeObj buildTree(String transactionFile)
	{
		Node root = new Node();
		// this.root = root;
		
		Map<Integer, Node> headerTableLast = new HashMap<Integer, Node>();
		
		Map<Integer, Integer> itemFreqs = countItemSupport(transactionFile);
		
		List<Integer> aList = new ArrayList<Integer>();
		BitSet itemsBelowSupport = new BitSet();
		for (Entry<Integer, Integer> pair: itemFreqs.entrySet())
		{
			if (pair.getValue() >= 0)
				aList.add(pair.getKey());
			else
				itemsBelowSupport.set(pair.getKey());
		}
		Collections.sort(aList, new FListComparator(itemFreqs));

		
//		System.out.println("fList " + fList.size() + ": " + fList);
		// System.out.println("itemFreqs: " + itemFreqs);
		
		Map<Integer, Node> headerTable = new LinkedHashMap<Integer, Node>();
		TreeMethods.initialiseLinkedHashMap(headerTable, aList);
		
		int transactionNum = 0;
		Map<Integer, Integer> fListMap = TreeMethods.listToMap(aList);
		for (CsvReader reader = new CsvReader(transactionFile); reader.hasNext();)
		{
			List<Integer> transaction = new ArrayList<Integer>(reader.readIgnore(itemsBelowSupport));
			if (!transaction.isEmpty())
			{
				transactionNum++;
				TreeMethods.processTransaction(transaction, 1, root, headerTable, headerTableLast, fListMap);
			}
		}

		BitSet rBitSet = createListsMinFreq(itemFreqs, aList, (int) Math.ceil(0.15 * transactionNum));
		System.out.println("maxFreq: " + (int) Math.ceil(0.15 * transactionNum) + " rBitSet " + rBitSet.cardinality() + ": " + rBitSet);
		
		return new TreeObj(headerTable, root, itemFreqs, rBitSet, transactionNum);
	}
	private class TreeObj {
		Map<Integer, Node> headerTable;
		Node root;
		Map<Integer, Integer> itemFreqs;
		int transactionNum;
		BitSet rares;
		TreeObj(Map<Integer, Node> headerTable, Node root, Map<Integer, Integer> itemFreqs, BitSet rares, int transactionNum)
		{
			this.headerTable = headerTable;
			this.root = root;
			this.itemFreqs = itemFreqs;
			this.transactionNum = transactionNum;
			this.rares = rares;
		}
	}

	public Map<Integer, Integer> countItemSupport(String transactionFile) {
		Map<Integer, Integer> itemFreqs = new HashMap<Integer, Integer>();

		for (CsvReader reader = new CsvReader(transactionFile); reader.hasNext(); ) {	// look through all transactions
			List<Integer> list = reader.read();
			for (int item : list) {					// count item frequency
				if (!itemFreqs.containsKey(item)) {
					itemFreqs.put(item, 1);
				} else {
					itemFreqs.put(item, itemFreqs.get(item)+1);
				}
			}
		}
//		System.out.println("ItemSetMining transactionCount: " + transactionCount + ", itemCount: " + uniqueItemCount);
		return itemFreqs;
	}

	private BitSet createListsMinFreq(Map<Integer, Integer> itemFreqs, List<Integer> aList, int minFreqItemsetSupport)
	{
		int splitIndex = -1;
		
		for (int i = 0; i < aList.size(); i++)
		{
			if (itemFreqs.get(aList.get(i)) < minFreqItemsetSupport)
			{
				splitIndex = i;
				break;
			}
		}
		
		BitSet rares = new BitSet();
		
		System.out.println(aList.subList(splitIndex, aList.size()));
		
		for (int i = splitIndex; i < aList.size(); i++)
		{
			rares.set(aList.get(i));
		}
		
		return rares;
	}

	double conf(int a, int ab)
	{
		return (double) ab / a;
	}
	
	double chisq(int a, int b, int ab, int transactions)
	{
		int nab = transactions - a - b + ab;
		
		double pa = (double) (a ) / transactions;
		double pb = (double) (b ) / transactions;
		double ipa = 1 - pa;
		double ipb = 1 - pb;
		
		double ea = transactions * pa * ipb;
		double eb = transactions * ipa * pb;
		double eab = transactions * pa * pb;
		double enab = transactions * ipa * ipb;
		
		return Math.pow((ea - a + ab), 2)/ea + Math.pow((eb - b + ab), 2)/eb + Math.pow((eab - ab), 2)/eab + Math.pow((enab - nab), 2)/enab;
	}
	
	double lift(int a, int b, int ab, int transactions)
	{
		return (double) ab * transactions / a / b;
	}
	
	double allConf(int a, int b, int ab)
	{
		return (double)ab / Math.max(a, b);
	}
	
	double coherence(int a, int b, int ab)
	{
		return (double) ab / (a + b - ab);
	}
	
	double cosine(int a, int b, int ab)
	{
		return ab / Math.sqrt(a * b);
	}
	
	double kulc(int a, int b, int ab)
	{
		return (double) ab/2*((double) 1/a + (double) 1/b);
	}
	
	double maxConf(int a, int b, int ab)
	{
		if (a > b)
			return (double) ab/b;
		else
			return (double) ab/a;
	}
}
